# -*- coding: utf-8 -*-
"""
@Time ： 2023/9/10 20:59
@Auth ： y.h
@File ：data_cleansing.py
@IDE ：PyCharm
@Motto：The sooner you start to code, the longer the program will take. —— Roy Carlson
"""
# todo 给这个类的代码写注释好累,不想写,将就着看吧
# 数据清洗
# 数据清洗为选择调用,不计入程序运行时间,主要是这个类的for循环确实不好消减,如果强行消减,调方法的时候会很麻烦
__all__ = ["data_cleansing",
           "data_type_detection",
           "check_missing_value",
           "type_select_values"]

import math

import pandas
import pandas as pd

from pythonProjectTemplate.Entity.DataEntity.price_entity import price_entity
from pythonProjectTemplate.Entity.DataEntity.cost_entity import cost_entity
from pythonProjectTemplate.Entity.DataEntity.BOM_entity import bom_entity
from pythonProjectTemplate.Entity.SystemEntity.del_row_or_col_properties import 删除行或列配置文件
from pythonProjectTemplate.Entity.SystemEntity.entity_load import get_entity_setFunc, get_entity_getFunc
from pythonProjectTemplate.Entity.SystemEntity.re_df_name import 更改列名配置文件
from pythonProjectTemplate.Entity.SystemEntity.remove_missing_value_properties import 数据清洗配置文件
from pythonProjectTemplate.Entity.SystemEntity.create_or_update_line_properties import 行操作配置文件
from pythonProjectTemplate.Entity.SystemEntity.calculating_multiple_columns_entity import 多列运算配置文件
from pythonProjectTemplate.Entity.Enum.python_type_enum import type_enum
from pythonProjectTemplate.Entity.SystemEntity.data_duplicate_checking_entity import 数据查重配置文件
from pythonProjectTemplate.Entity.SystemEntity.reset_col_index_properties import 指定列索引配置文件
from pythonProjectTemplate.Utils.object_list_conversion import object_to_list

from collections import Counter
import numpy as np

import warnings


# 1.数据列命名 -> 数据加载时完成
# 2.筛查重复值
# 3.缺失值处理 3.1 缺失值删除, 3.2 缺失值填充
# 4 增删改查


def data_cleansing(loaded_data):
    pass


# 类型检测

def data_type_detection(data_dict: dict):
    wb_dict_type_list = dict()
    for wb_key in data_dict:

        wb_dict = data_dict.get(wb_key)

        sheet_dict_type_list = dict()
        for sheet_key in wb_dict:

            type_list = []
            sheet_dict = wb_dict.get(sheet_key)
            for primary_key in sheet_dict:
                obj = sheet_dict.get(primary_key)
                data_list = object_to_list(obj)
                print(data_list)
                d_type = []
                for data in data_list:
                    d_type.append(str(type(data)))
                type_list.append(d_type)
            sheet_dict_type_list.update({sheet_key: type_list})
        wb_dict_type_list.update({wb_key: sheet_dict_type_list})

    wb_type_dict = dict()
    for wb_key in wb_dict_type_list:
        sheet_type_dict = dict()
        for sheet_key in wb_dict_type_list.get(wb_key):

            type_list_2 = wb_dict_type_list.get(wb_key).get(sheet_key)
            print(sheet_key)
            init_type_dict = dict()
            for index in range(0, len(type_list_2[1])):
                init_type_dict.update({index + 1: []})

            for one_type_list in type_list_2:
                index_1 = 0

                for data_type in one_type_list:

                    index_1 += 1
                    # 初始化
                    if len(init_type_dict.get(index_1)) == 0:
                        init_type_dict.get(index_1).append({data_type: 1})
                    else:
                        value_list = init_type_dict.get(index_1)

                        value: dict
                        for value in value_list:
                            # todo 2023/9/25 22:17 这里有问题 -> 已解决
                            # print("-------")
                            # any(print(d.title()) for d in value_list)
                            if data_type in value.keys():
                                data = value.get(data_type)
                                data += 1
                                value[data_type] = data
                            else:
                                value[data_type] = 1

            sheet_type_dict.update({sheet_key: init_type_dict})
        wb_type_dict.update({wb_key: sheet_type_dict})
    for wb in wb_type_dict:

        for sheet in wb_type_dict.get(wb):
            print(f"\033[1;32;33m工作簿\"{wb}\"的工作表\"{sheet}\"的类型统计为:\033[0m")

            for value_2 in wb_type_dict.get(wb).get(sheet):
                any(print(f"列\"{value_2}\"的类型为{d}", end="\t") for d in
                    wb_type_dict.get(wb).get(sheet).get(value_2))
                print()


# 空值检测
def check_missing_value(data_dict: dict):
    wb_dict = dict()
    for wb in data_dict:
        sheet_dict = dict()
        for sheet in data_dict.get(wb):
            d_dict = dict()
            for data in data_dict.get(wb).get(sheet):
                data_list = object_to_list(data_dict.get(wb).get(sheet).get(data))
                index = 0
                for d in data_list:

                    if type(d) == float and math.isnan(float(d)):
                        print("_______")
                        data_list[index] = f"->{d}"

                        d_dict.update({data: data_list})
                        sheet_dict.update({sheet: d_dict})
                    index += 1
        wb_dict.update({wb: sheet_dict})

    for wb_1 in wb_dict:

        if len(wb_dict.get(wb_1).keys()) > 0:
            for sheet_1 in wb_dict.get(wb_1):
                print(f"\033[1;32;33m工作簿\"{wb_1}\"的工作表\"{sheet_1}\"的空值行为:\033[0m")
                for data_1 in wb_dict.get(wb_1).get(sheet_1):
                    print(f"键为\"{data_1}\"的数据为:\n{wb_dict.get(wb_1).get(sheet_1).get(data_1)}")
        else:
            print(f"{wb_1}工作簿下无空值!")


# 根据主键删除空值行
def remove_missing_value(data_dict: dict, properties: list[数据清洗配置文件], flag=False):
    print(properties)
    if len(properties) > 0:
        for pro in properties:
            # 不知道python会不会报空指针错误,也不想试,写个短路运算,反正也没损失
            if len(pro.get_wb_name()) > 0 and pro.get_wb_name() in data_dict.keys():
                if len(pro.get_sheet_name()) > 0 and pro.get_sheet_name() in data_dict.get(pro.get_wb_name()).keys():

                    if len(pro.get_primary_key()) > 0:

                        index_0 = 0

                        for primary_key in pro.get_primary_key():

                            index_0 += 1

                            if primary_key in data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).keys():

                                if flag:

                                    get_func_list = get_entity_getFunc(
                                        data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(primary_key))
                                    set_func_list = get_entity_setFunc(
                                        data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(primary_key))

                                    index_2 = 0
                                    for g_func, s_func in zip(get_func_list, set_func_list):
                                        get_func = getattr(
                                            data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(primary_key),
                                            g_func)
                                        set_func = getattr(
                                            data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(primary_key),
                                            s_func
                                        )
                                        data = get_func(
                                            data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(primary_key))

                                        if type(data) == float and math.isnan(float(data)):
                                            set_func(data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(
                                                primary_key), pro.get_update_value_list()[index_2])
                                            print(
                                                f"主键{primary_key}的值{data},已更改为{pro.get_update_value_list()[index_2]}")

                                            index_2 += 1
                                else:

                                    data_list = object_to_list(
                                        data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).get(primary_key))
                                    index_1 = 0

                                    for d in data_list:
                                        index_1 += 1
                                        if type(d) == float and math.isnan(float(d)):
                                            data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).pop(primary_key)
                                            print(f"主键为{primary_key}的下标索引{index_1},当前行已删除!")
                                        elif index_1 == len(data_list):
                                            print(f"{primary_key}主键值无空值")
                                            exit(True)
                    else:
                        print(f"主键参数未配置,请检查!")
                        exit(True)
    else:
        print("未配置删除值必要参数")
        exit(True)
    return data_dict


# 根据主键更改空值
def primary_key_update_missing_value(data_dict: dict, properties: list[数据清洗配置文件]):
    return remove_missing_value(data_dict, properties, True)


# 新增行
def create_line(data_dict: dict, properties: list[行操作配置文件]):
    properties_list = []
    properties_list.extend(properties)
    for pro in properties_list:
        if pro.get_wb_name() in data_dict.keys():
            if pro.get_sheet_name() in data_dict.get(pro.get_wb_name()).keys():
                for primary_key in pro.get_primary_key_and_values():
                    data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()).update(
                        {primary_key: pro.get_primary_key_and_values().get(primary_key)})

    return data_dict


# 根据列类型查值
def type_select_values(clo_key: str, t_e: type_enum, data_dict: dict, properties: list[数据清洗配置文件]):
    properties_list = []
    properties_list.extend(properties)
    for pro in properties_list:
        if pro.get_wb_name() in data_dict.keys():
            if pro.get_sheet_name() in data_dict.get(pro.get_wb_name()).keys():
                print(pro.get_sheet_name())
                for data_entity_key in data_dict.get(pro.get_wb_name()).get(pro.get_sheet_name()):
                    print(data_entity_key)


# 根据主键查值


# 根据特定值查值

# 重复值筛查

# 替换行
# 懒得写,直接用新增行就行,注意主键的用法就好


# 数据表合并
def table_merge(df_list: list, link_list: list, link_func="inner"):
    if len(df_list) == 1:
        return df_list[0]
    elif len(df_list) > 1:
        index_list = df_list[0]
        df_list_length = len(df_list)
        for i in range(1, df_list_length):
            index_list = pd.merge(index_list, df_list[i], on=link_list[i - 1], how=link_func)

        return index_list


# 判断是否合并

# 指定列筛查
def specifies_column_screening(df, properties: str):
    return df.query(properties)


# 指定值筛查

def specifies_value_screening():
    pass


# 数据表查重
def data_deduplication(data_dict, properties: list[数据查重配置文件]):
    df = pandas.DataFrame()
    for pro in properties:
        if type(pro) is not None:
            if pro.get_file_name() in data_dict.keys():
                if pro.get_sheet_name() in data_dict.get(pro.get_file_name()):
                    if len(pro.get_col_index()) > 0:
                        sheet_name = data_dict.get(pro.get_file_name()).get(pro.get_sheet_name())

                        sheet_name = sheet_name[sheet_name.duplicated(subset=pro.get_col_index(), keep=False)]

                        return sheet_name

                    else:
                        sheet_name = data_dict.get(pro.get_file_name()).get(pro.get_sheet_name())

                        sheet_name = sheet_name[sheet_name.duplicated(subset=sheet_name.columns.values, keep=False)]

                        return sheet_name
            elif type(data_dict) == type(df):
                if len(pro.get_col_index()) > 0:
                    sheet_name = data_dict[data_dict.duplicated(subset=pro.get_col_index(), keep=False)]

                    return sheet_name

                else:

                    sheet_name = data_dict[data_dict.duplicated(subset=data_dict.columns.values, keep=False)]

                    return sheet_name

        else:
            print("配置文件为空!")
            exit(True)


# 表行列转置
def col_row_conversion(df):
    df = df.copy()
    return df.T


# 修改列名
def re_col_name(df, properties: 更改列名配置文件):
    return df.rename(columns=properties.get_rename_dict())


# 判断某列是否包含某些值,并返回该列
def judge_col_is_value(df, col_value: str, value_list: list):
    return df.loc[df[col_value].isin(value_list)]


# 多列运算
def calculating_multiple_columns(data_and_properties: 多列运算配置文件):
    df = data_and_properties.get_data_dict()

    operator = data_and_properties.get_operator()
    try:
        if type(df) != type(pandas.DataFrame()):
            df = df.get(data_and_properties.get_file_name()).get(data_and_properties.get_sheeet_name())
    except Exception:
        print("多列运算数据集配置错误,需要%s类型,或{文件名:{表单名:df}}类型的数据,请检查实参类型!" % (
            type(pandas.DataFrame)))
        exit(True)

    cal = None
    if operator.value == "add":
        cal = "+"
    elif operator.value == "subtract":
        cal = "-"
    elif operator.value == "multiply":
        cal = "*"
    elif operator.value == "Dividing":
        cal = "/"
    elif operator.value == "remainder":
        cal = "%"

    for arg_list_1 in data_and_properties.get_arg_list_1():
        for arg_list_2 in data_and_properties.get_arg_list_2():
            # print(f"lambda x:x[{arg_list_1}]{cal} x[{arg_list_2}]")
            df[f"{arg_list_1}_{operator.value}_{arg_list_2}"] = df.apply(
                eval(f"lambda x:x['{arg_list_1}']{cal} x['{arg_list_2}']"), axis=1)

    return df


# 多列多行求和
def col_or_line_sum(df, range_start: int, range_end: int, result_name: str, flag=False):
    """
            flag = false = col
            """

    df = df.iloc[:, range_start:range_end].copy()
    if not flag:
        if len(result_name) < 1:
            result_name = "col_result"

            df.loc[result_name, :] = df.sum(axis=0)
            # df[result_name] = df.apply(lambda x: x.sum(), axis=1)
        else:

            df.loc[result_name, :] = df.sum(axis=0)
    else:
        if len(result_name) < 1:
            result_name = "line_result"
            df[result_name] = df.iloc[:, range_start:range_end].sum(axis=1)

        else:
            df[result_name] = df.iloc[:, range_start:range_end].sum(axis=1)

    return df


# 按照条件删除行或列
def condition_del_col_or_line(df: pandas.DataFrame, condition_str: str, is_line=True):
    df = df.copy()
    try:
        if is_line:
            df.drop(df[eval(condition_str)].index, inplace=True)
        else:
            df.drop(columns=condition_str, inplace=True)
    except Exception as e:
        print(e)
    finally:
        return df


# 新增列
def update_col(df, col_index, col_name, col_data):
    return df.insert(col_index, col_name, col_data)


# 读取某列的所有值
def load_col_values(df, col_index):
    df.copy()
    return df.iloc[:, col_index]


# 指定行或列的删除
def del_row_or_col(properties: 删除行或列配置文件):
    if type(properties) != type(删除行或列配置文件()):
        print("配置文件参数错误!")
        exit(True)
    else:
        if len(properties.get_col_index()) > 0 or len(properties.get_col_name()) > 0 or len(
                properties.get_row_name()) > 0 or len(properties.get_row_index()) > 0:
            df = properties.get_df().copy()
            if len(properties.get_col_index()) > 0:
                df.drop(index=properties.get_df().index[properties.get_col_index()], inplace=True, axis=1)
            elif len(properties.get_col_name()) > 0:
                df.drop(index=properties.get_col_name(), inplace=True, axis=1)
            elif len(properties.get_row_name()) > 0:
                df.drop(index=properties.get_row_name(), inplace=True)
            elif len(properties.get_row_index()) > 0:
                df.drop(index=properties.get_df().index[properties.get_row_index()], inplace=True)
            else:
                print("删除行或列错误!")
                return Exception()
            return df
        else:
            print(properties.get_row_index())
            print("删除行或列的配置文件为空!")
            exit(True)


def reset_col_index(properties: 指定列索引配置文件):
    if type(properties) != type(指定列索引配置文件()):
        print("重置列索引,配置文件参数错误!")
        exit(True)
    elif type(properties.get_df()) != type(pandas.DataFrame()):
        print(f"数据集类型错误!需要{type(pandas.DataFrame())},参数为{type(properties.get_df())}")
        exit(True)
    elif properties.get_col_index() is None:
        print("列索引配置为空!")
        exit(True)
    else:
        df = properties.get_df().copy()
        col_index = properties.get_col_index()
        c_list = df.values.tolist()[col_index]
        df.columns = c_list
        df.drop([col_index], inplace=True)
        return df
